bitkeeper revision 1.1159.135.1 (417fd3b8e-uijqG_Stg0EouCeWPftA)

Add network bandwidth adaptive rate limiting to migration and
timestamped info messages.

diff --git a/tools/libxc/xc_io.c b/tools/libxc/xc_io.c
index 4615092e97b94ac452c8b2cd8179e4c2ae764c71..5589483f1088f5182df316c6d34eabf975754896 100644 (file)
--- a/tools/libxc/xc_io.c
+++ b/tools/libxc/xc_io.c
@@ -1,4 +1,13 @@
 #include "xc_io.h"
+#include <sys/time.h>
+
+void xcio_timestamp(XcIOContext *ctxt, const char *msg){
+  struct timeval tv;
+
+  gettimeofday(&tv, NULL);
+  if (msg[0] != '\b' && msg[0] != '\r')
+      fprintf(stdout, "[%08ld.%06ld] ", tv.tv_sec, tv.tv_usec);
+}
 
 void xcio_error(XcIOContext *ctxt, const char *msg, ...){
   va_list args;
@@ -15,8 +24,10 @@ void xcio_info(XcIOContext *ctxt, const char *msg, ...){
 
   if(0 && !(ctxt->flags & XCFLAGS_VERBOSE)) return;
   va_start(args, msg);
+  xcio_timestamp(ctxt, msg);
   vfprintf(stdout, msg, args); fprintf(stdout, "\n");
   IOStream_vprint(ctxt->info, msg, args);
+  fflush(stdout);
   va_end(args);
 }
 
@@ -25,6 +36,7 @@ void xcio_debug(XcIOContext *ctxt, const char *msg, ...){
 
   if(0 && !(ctxt->flags & XCFLAGS_DEBUG)) return;
   va_start(args, msg);
+  xcio_timestamp(ctxt, msg);
   vfprintf(stdout, msg, args); fprintf(stdout, "\n");
   IOStream_vprint(ctxt->info, msg, args);
   va_end(args);

diff --git a/tools/libxc/xc_linux_restore.c b/tools/libxc/xc_linux_restore.c
index 247e972fd6d61229c533851a6f51e3474d95bf44..2b3df66dd553bd4ccfeffcb2587f5b390c1ff347 100644 (file)
--- a/tools/libxc/xc_linux_restore.c
+++ b/tools/libxc/xc_linux_restore.c
@@ -130,6 +130,8 @@ int xc_linux_restore(int xc_handle, XcIOContext *ioctxt)
     /* used by debug verify code */
     unsigned long buf[PAGE_SIZE/sizeof(unsigned long)];
 
+    xcio_info(ioctxt, "xc_linux_restore start\n");
+
     if ( mlock(&ctxt, sizeof(ctxt) ) )
     {
         /* needed for when we do the build dom0 op, 
@@ -194,7 +196,7 @@ int xc_linux_restore(int xc_handle, XcIOContext *ioctxt)
     }
     
     ioctxt->domain = dom;
-    printf("Created domain %ld\n",dom);
+    xcio_info(ioctxt, "Created domain %ld\n",dom);
 
     /* Get the domain's shared-info frame. */
     op.cmd = DOM0_GETDOMAININFO;
@@ -433,7 +435,7 @@ int xc_linux_restore(int xc_handle, XcIOContext *ioctxt)
         n+=j; /* crude stats */
     }
 
-    DPRINTF("Received all pages\n");
+    xcio_info(ioctxt, "Received all pages\n");
 
     /*
      * Pin page tables. Do this after writing to them as otherwise Xen
@@ -473,7 +475,8 @@ int xc_linux_restore(int xc_handle, XcIOContext *ioctxt)
 
     if ( finish_mmu_updates(xc_handle, mmu) ) goto out;
 
-    xcio_info(ioctxt, "\b\b\b\b100%%\nMemory reloaded.\n");
+    xcio_info(ioctxt, "\b\b\b\b100%%\n");
+    xcio_info(ioctxt, "Memory reloaded.\n");
 
     /* Get the list of PFNs that are not in the psuedo-phys map */
     {
@@ -647,7 +650,9 @@ int xc_linux_restore(int xc_handle, XcIOContext *ioctxt)
         xcio_error(ioctxt, "Bad LDT base or size");
         goto out;
     }
-   
+
+    xcio_info(ioctxt, "Domain ready to be built.\n");
+
     op.cmd = DOM0_BUILDDOMAIN;
     op.u.builddomain.domain   = (domid_t)dom;
     op.u.builddomain.ctxt = &ctxt;
@@ -661,6 +666,7 @@ int xc_linux_restore(int xc_handle, XcIOContext *ioctxt)
 
     if ( ioctxt->flags & XCFLAGS_CONFIGURE )
     {
+        xcio_info(ioctxt, "Domain ready to be unpaused\n");
         op.cmd = DOM0_UNPAUSEDOMAIN;
         op.u.unpausedomain.domain = (domid_t)dom;
         rc = do_dom0_op(xc_handle, &op);

diff --git a/tools/libxc/xc_linux_save.c b/tools/libxc/xc_linux_save.c
index 504a1223e2871d0a695b7c5aeada86934ddc9359..7e0ab5c502ac2069931b7aaf7bfab8fad5e7448c 100644 (file)
--- a/tools/libxc/xc_linux_save.c
+++ b/tools/libxc/xc_linux_save.c
@@ -9,9 +9,12 @@
 #include <sys/time.h>
 #include "xc_private.h"
 #include <asm-xen/suspend.h>
+#include <time.h>
 
 #define BATCH_SIZE 1024   /* 1024 pages (4MB) at a time */
 
+#define MAX_MBIT_RATE 500
+
 #define DEBUG  0
 #define DDEBUG 0
 
@@ -138,6 +141,80 @@ static long long tv_delta( struct timeval *new, struct timeval *old )
         (new->tv_usec - old->tv_usec);
 }
 
+
+#define START_MBIT_RATE ioctxt->resource
+
+static int mbit_rate, ombit_rate = 0;
+static int burst_time_us = -1;
+
+#define MBIT_RATE mbit_rate
+#define BURST_BUDGET (100*1024)
+
+/* 
+   1000000/((100)*1024*1024/8/(100*1024))
+   7812
+   1000000/((100)*1024/8/(100))
+   7812
+   1000000/((100)*128/(100))
+   7812
+   100000000/((100)*128)
+   7812
+   100000000/128
+   781250
+ */
+#define RATE_TO_BTU 781250
+#define BURST_TIME_US burst_time_us
+
+static int xcio_ratewrite(XcIOContext *ioctxt, void *buf, int n){
+    static int budget = 0;
+    static struct timeval last_put = { 0 };
+    struct timeval now;
+    struct timespec delay;
+    long long delta;
+    int rc;
+
+    if (START_MBIT_RATE == 0)
+       return xcio_write(ioctxt, buf, n);
+    
+    budget -= n;
+    if (budget < 0) {
+       if (MBIT_RATE != ombit_rate) {
+           BURST_TIME_US = RATE_TO_BTU / MBIT_RATE;
+           ombit_rate = MBIT_RATE;
+           xcio_info(ioctxt,
+                     "rate limit: %d mbit/s burst budget %d slot time %d\n",
+                     MBIT_RATE, BURST_BUDGET, BURST_TIME_US);
+       }
+       if (last_put.tv_sec == 0) {
+           budget += BURST_BUDGET;
+           gettimeofday(&last_put, NULL);
+       } else {
+           while (budget < 0) {
+               gettimeofday(&now, NULL);
+               delta = tv_delta(&now, &last_put);
+               while (delta > BURST_TIME_US) {
+                   budget += BURST_BUDGET;
+                   last_put.tv_usec += BURST_TIME_US;
+                   if (last_put.tv_usec > 1000000) {
+                       last_put.tv_usec -= 1000000;
+                       last_put.tv_sec++;
+                   }
+                   delta -= BURST_TIME_US;
+               }
+               if (budget > 0)
+                   break;
+               delay.tv_sec = 0;
+               delay.tv_nsec = 1000 * (BURST_TIME_US - delta);
+               while (delay.tv_nsec > 0)
+                   if (nanosleep(&delay, &delay) == 0)
+                       break;
+           }
+       }
+    }
+    rc = IOStream_write(ioctxt->io, buf, n);
+    return (rc == n ? 0 : rc);
+}
+
 static int print_stats( int xc_handle, u32 domid, 
                         int pages_sent, xc_shadow_control_stats_t *stats,
                         int print )
@@ -168,12 +245,20 @@ static int print_stats( int xc_handle, u32 domid,
 
     if ( print )
         printf("delta %lldms, dom0 %d%%, target %d%%, sent %dMb/s, "
-               "dirtied %dMb/s\n",
+               "dirtied %dMb/s %ld pages\n",
                wall_delta, 
                (int)((d0_cpu_delta*100)/wall_delta),
                (int)((d1_cpu_delta*100)/wall_delta),
-               (int)((pages_sent*PAGE_SIZE*8)/(wall_delta*1000)),
-               (int)((stats->dirty_count*PAGE_SIZE*8)/(wall_delta*1000)));
+               (int)((pages_sent*PAGE_SIZE)/(wall_delta*(1000/8))),
+               (int)((stats->dirty_count*PAGE_SIZE)/(wall_delta*(1000/8))),
+               stats->dirty_count);
+
+    if (((stats->dirty_count*PAGE_SIZE)/(wall_delta*(1000/8))) > mbit_rate) {
+       mbit_rate = (int)((stats->dirty_count*PAGE_SIZE)/(wall_delta*(1000/8)))
+           + 50;
+       if (mbit_rate > MAX_MBIT_RATE)
+           mbit_rate = MAX_MBIT_RATE;
+    }
 
     d0_cpu_last  = d0_cpu_now;
     d1_cpu_last  = d1_cpu_now;
@@ -198,14 +283,15 @@ static int write_vmconfig(XcIOContext *ioctxt){
 }
 
 static int analysis_phase( int xc_handle, u32 domid, 
-                           int nr_pfns, unsigned long *arr )
+                           int nr_pfns, unsigned long *arr, int runs )
 {
     long long start, now;
     xc_shadow_control_stats_t stats;
+    int j;
 
     start = llgettimeofday();
 
-    while ( 0 )
+    for (j = 0; j < runs; j++)
     {
         int i;
 
@@ -213,9 +299,9 @@ static int analysis_phase( int xc_handle, u32 domid,
                            DOM0_SHADOW_CONTROL_OP_CLEAN,
                            arr, nr_pfns, NULL);
         printf("#Flush\n");
-        for ( i = 0; i < 100; i++ )
+        for ( i = 0; i < 40; i++ )
         {     
-            usleep(10000);     
+            usleep(50000);     
             now = llgettimeofday();
             xc_shadow_control( xc_handle, domid, 
                                DOM0_SHADOW_CONTROL_OP_PEEK,
@@ -345,6 +431,10 @@ int xc_linux_save(int xc_handle, XcIOContext *ioctxt)
 
     int needed_to_fix = 0;
     int total_sent    = 0;
+
+    MBIT_RATE = START_MBIT_RATE;
+
+    xcio_info(ioctxt, "xc_linux_save start %d\n", domid);
     
     if (mlock(&ctxt, sizeof(ctxt))) {
         xcio_perror(ioctxt, "Unable to mlock ctxt");
@@ -440,7 +530,6 @@ int xc_linux_save(int xc_handle, XcIOContext *ioctxt)
         }
 
         last_iter = 0;
-        sent_last_iter = 1<<20; /* 4GB of pages */
     } else{
        /* This is a non-live suspend. Issue the call back to get the
         domain suspended */
@@ -455,6 +544,7 @@ int xc_linux_save(int xc_handle, XcIOContext *ioctxt)
        }
 
     }
+    sent_last_iter = 1<<20; /* 4GB of pages */
 
     /* calculate the power of 2 order of nr_pfns, e.g.
        15->4 16->4 17->5 */
@@ -493,7 +583,7 @@ int xc_linux_save(int xc_handle, XcIOContext *ioctxt)
 
     }
 
-    analysis_phase( xc_handle, domid, nr_pfns, to_skip );
+    analysis_phase( xc_handle, domid, nr_pfns, to_skip, 0 );
 
     /* We want zeroed memory so use calloc rather than malloc. */
     pfn_type = calloc(BATCH_SIZE, sizeof(unsigned long));
@@ -744,14 +834,14 @@ int xc_linux_save(int xc_handle, XcIOContext *ioctxt)
    
                     } /* end of page table rewrite for loop */
       
-                    if ( xcio_write(ioctxt, page, PAGE_SIZE) ){
+                    if ( xcio_ratewrite(ioctxt, page, PAGE_SIZE) ){
                         xcio_error(ioctxt, "Error when writing to state file (4)");
                         goto out;
                     }
       
                 }  /* end of it's a PT page */ else {  /* normal page */
 
-                    if ( xcio_write(ioctxt, region_base + (PAGE_SIZE*j), 
+                    if ( xcio_ratewrite(ioctxt, region_base + (PAGE_SIZE*j), 
                                      PAGE_SIZE) ){
                         xcio_error(ioctxt, "Error when writing to state file (5)");
                         goto out;
@@ -801,7 +891,8 @@ int xc_linux_save(int xc_handle, XcIOContext *ioctxt)
         if ( live )
         {
             if ( 
-                /* ( sent_this_iter > (sent_last_iter * 0.95) ) || */
+                ( ( sent_this_iter > sent_last_iter ) &&
+                 (mbit_rate == MAX_MBIT_RATE ) ) ||
                 (iter >= max_iters) || 
                 (sent_this_iter+skip_this_iter < 50) || 
                 (total_sent > nr_pfns*max_factor) )
@@ -816,11 +907,11 @@ int xc_linux_save(int xc_handle, XcIOContext *ioctxt)
                    goto out;
                }
 
-               printf("SUSPEND flags %08lx shinfo %08lx eip %08lx esi %08lx\n", 
-                      op.u.getdomaininfo.flags, op.u.getdomaininfo.shared_info_frame,
-                      ctxt.cpu_ctxt.eip, ctxt.cpu_ctxt.esi );
-
-
+               xcio_info(ioctxt,
+                          "SUSPEND flags %08lx shinfo %08lx eip %08lx "
+                          "esi %08lx\n", op.u.getdomaininfo.flags,
+                          op.u.getdomaininfo.shared_info_frame,
+                          ctxt.cpu_ctxt.eip, ctxt.cpu_ctxt.esi );
             } 
 
             if ( xc_shadow_control( xc_handle, domid, 

diff --git a/tools/python/xen/xm/migrate.py b/tools/python/xen/xm/migrate.py
index 070905bc1146bb28d70a0436b82b93b1d81daf7b..147c0c4d08d6a26c1d4d9c9e1d74d4081554f92c 100644 (file)
--- a/tools/python/xen/xm/migrate.py
+++ b/tools/python/xen/xm/migrate.py
@@ -26,7 +26,7 @@ gopts.opt('live', short='l',
           fn=set_true, default=0,
           use="Use live migration.")
 
-gopts.opt('resource', short='r',
+gopts.opt('resource', short='r', val='MBIT',
           fn=set_int, default=0,
           use="Set level of resource usage for migration.")